package com.walker.service.meta;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.dao.DuplicateKeyException;
import org.springframework.dao.EmptyResultDataAccessException;
import org.springframework.jdbc.core.BatchPreparedStatementSetter;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;

import com.fasterxml.uuid.Generators;
import com.jayway.jsonpath.JsonPath;
import com.walker.bean.MetaColBean;
import com.walker.bean.MetaPrtBean;
import com.walker.bean.MetaTabBean;
import com.walker.util.AesEncryptUtils;
import com.walker.util.CommonUtils;
import com.walker.util.ConstantUtil;
import com.walker.util.DateUtils;
import com.walker.util.JdbcUtil;
import com.walker.util.RestTemplateUtil;

import net.sf.json.JSONObject;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

import org.apache.commons.lang3.StringUtils;


/**
 * 数据源增加tidb，url需要控制填写具体的库名
 * 计算引擎源，需增加tidb引擎源
 * @author gaolei
 *
 */
@Component
public class AssetMetaDataCollectServ {
	@Autowired
	@Qualifier("dmJdbcTemplate")
	private JdbcTemplate gJdbc;
	
	
	@Scheduled(cron = "${syncmeta.task.interval.time:0 0 05 * * *}")
	void syncMetaTask() {
		try {
			System.out.println(syncMeta(""));
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		System.err.println();
		System.err.println(DateUtils.getCurrentDateTime() + " >> >> >> Synchronize metadata");
		System.err.println();
	}
	
	public Map<String, Object> syncMeta(String projectCode) throws Exception {
		try {
			gJdbc.queryForMap("select 1 from t_asset_meta_sync_log where status ='0' and sync_time > DATE_FORMAT(DATE_SUB(NOW(),interval 40 minute),'%Y-%m-%d %H:%i:%s') limit 1");
			throw new Exception("为保证生产环境性能,一小时内,只允许手动同步一次");
		} catch (EmptyResultDataAccessException e) {
		}
		try {
			List<Map<String, Object>> dsData = gJdbc.queryForList("select id,data_source_name from t_meta_data_source");
			String sql = "insert into t_asset_meta_sync_conf(ds_id,ds_name,on_off,oper_time) values(?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dsData.get(i).get("id"));
					ps.setObject(2, dsData.get(i).get("data_source_name"));
					ps.setObject(3, "on");
					ps.setObject(4, DateUtils.getCurrentDateTime());
				}
				
				@Override
				public int getBatchSize() {
					return dsData.size();
				}
			});
		} catch (DuplicateKeyException  e) {
		} catch (Exception  e) {
			e.printStackTrace();
		}
		
		try {
			//hive已单独同步到中台，此处只做表合并
			List<Map<String, Object>> logList = new ArrayList<>();
			logList.add(mergeHiveMeta(projectCode));
			saveLog(logList);
			
			List<Map<String, Object>> connectInfo = null;
			if (StringUtils.isBlank(projectCode)) {
				System.err.println("同步所有数据源元数据");
				connectInfo = gJdbc.queryForList("select ds.data_source_type ,ds.data_source_name ,ds.id as dsId ,group_concat(concat_ws('@~',param.param_key,param.param_value) order by param.param_key desc) param "
						+ "from t_meta_data_source ds ,t_meta_data_source_param param "
						+ "where param.id = ds.id and param.param_key in ('jdbcuser','jdbcurl','jdbcpassword','http_urls','user','password') and ds.id not in (select ds_id from t_asset_meta_sync_conf where on_off ='off') "
						+ "group by ds.data_source_type ,ds.data_source_name ,ds.id ");
			} else {
				System.err.println("同步 [" + projectCode + "] 项目元数据");
				connectInfo = gJdbc.queryForList("select p.project_code ,ds.data_source_type ,ds.data_source_name ,ds.id as dsId ,group_concat(concat_ws('@~',param.param_key,param.param_value) order by param.param_key desc) param  "
						+ "from t_meta_data_source ds ,t_meta_data_source_param param ,t_meta_project p ,t_meta_project_ds pds "
						+ "where p.id = pds.project_id and ds.id = pds.ds_id and param.id = ds.id and p.project_code =? and param.param_key in ('jdbcuser','jdbcurl','jdbcpassword','http_urls','user','password') and ds.id not in (select ds_id from t_asset_meta_sync_conf where on_off ='off') "
						+ "group by p.project_code ,ds.data_source_type ,ds.data_source_name ,ds.id",projectCode);
			}
			
			//不重复执行记录使用
			Map<String, Object> execResult  = new HashMap<>();
			//写日志记录执行结果使用
			Map<String, Object> result = new HashMap<>();
			//存放对象归属关系使用
			Map<String, Object> structMap = new HashMap<>();
			List<Map<String, Object>> struct = new ArrayList<>();
			for (int i = 0; i < connectInfo.size(); i++) {
				logList = new ArrayList<>();
				Map<String,Object> info = connectInfo.get(i);
				String data_source_type = String.valueOf(info.get("data_source_type")).toLowerCase();
				info.put("data_source_type", data_source_type);
				String param = String.valueOf(info.get("param"));
				if (null == param || param.isEmpty()) {
					continue;
				}
				String[] paramArry = param.split(",");
				String userName = null, jdbcUrl = null,passWord = null;
				for (int p=0; p<paramArry.length; p++) {
					if (paramArry[p].split("@~").length < 2) {continue;};
					String key = paramArry[p].split("@~")[0];
					String value = paramArry[p].split("@~")[1];
					if ("jdbcuser".equals(key) || "user".equals(key)) {
						userName = value;
					} else if ("jdbcpassword".equals(key) || "password".equals(key)) {
						passWord = value;
					} else if ("jdbcurl".equals(key) || "http_urls".equals(key) || "es_url".equals(key)) {
						jdbcUrl = value;
					}
				}
				
				if (CommonUtils.isNull(jdbcUrl)) {continue;};
				String unique = data_source_type + ":" + JdbcUtil.parseJdbc(jdbcUrl, data_source_type).get("unique");
				Map<String,String> jdbcMap = JdbcUtil.parseJdbc(jdbcUrl, data_source_type);
				jdbcUrl = jdbcMap.get("url");
				info.put("userName", userName);
				info.put("jdbcUrl", jdbcUrl);
				info.put("passWord", passWord);
				
				structMap = new HashMap<String,Object>();
				structMap.put("ds_id", info.get("dsId"));
				structMap.put("jdbcUrl", jdbcUrl);
				structMap.put("database", jdbcMap.get("database"));
				structMap.put("data_source_type", data_source_type);
				struct.add(structMap);
				
				//数据源中，增加多个相同数据源，只收集一次
				if (execResult.containsKey(unique) && !execResult.get(unique).equals(false)) {
					continue;
				} else {
					execResult.put(unique, false);
				}
				
				result = new HashMap<>();
				switch (data_source_type) {
				case "mysql": 
					if (!userName.equals("root")) {
						break;
					}
					info.put("db_type", "mysql");
					result = syncTidbMeta(info);
					execResult.put(unique, result);
					break;
					
				case "tidb":
					if (!userName.equals("root")) {
						break;
					}
					info.put("db_type", "tidb");
					result = syncTidbMeta(info);
					execResult.put(unique, result);
					break;
					
				case "hive":
//					syncTidbMeta(info); 目前hive是按数仓做收集，不支持物理数据源收集。此处为数据保护伞提前hive元数据使用
					break;
					
				case "oracle":
					if (!userName.equals("root")) {
						break;
					}
					info.put("db_type", "oracle");
					result = syncOracleMeta(info);
					execResult.put(unique, result);
					break;
					
				case "presto":
//					syncTidbMeta(info);  数据源增加catalog区分
					break;
					
				case "clickhouse":
					if (!userName.equals("default")) {
						break;
					}
					info.put("db_type", "clickhouse");
					result = syncCKMeta(info);
					execResult.put(unique, result);
					break;

				case "elasticsearch":
					result = syncEsMeta(info);
					execResult.put(unique, result);
					break;

				default:
					break;
				}
				if (null != result && !result.isEmpty()) {
					System.out.println("完成:" + result);
					logList.add(result);
					saveLog(logList);
				}
			}
			matchStruct(struct);
			return execResult;
		} catch (Exception e) {
			e.printStackTrace();
		}
		return null;
	}
	
	public boolean analyzeMeta(String projectCode) throws Exception {
		List<Map<String, Object>> connectInfo = null;
		if (null == projectCode || "".equals(projectCode)) {
			connectInfo = gJdbc.queryForList("select ds.data_source_type ,ds.data_source_name ,ds.id as dsId ,group_concat(param.param_value order by param.param_key desc) param "
					+ "from t_meta_data_source ds ,t_meta_data_source_param param "
					+ "where param.id = ds.id and param.param_key in ('jdbcuser','jdbcurl','jdbcpassword') "
					+ "group by ds.data_source_type ,ds.data_source_name ,ds.id ");
		} else {
			connectInfo = gJdbc.queryForList("select p.project_code ,ds.data_source_type ,ds.data_source_name ,ds.id as dsId ,group_concat(param.param_value order by param.param_key desc) param  "
					+ "from t_meta_data_source ds ,t_meta_data_source_param param ,t_meta_project p ,t_meta_project_ds pds "
					+ "where p.id = pds.project_id and ds.id = pds.ds_id and param.id = ds.id and p.project_code =? and param.param_key in ('jdbcuser','jdbcurl','jdbcpassword') "
					+ "group by p.project_code ,ds.data_source_type ,ds.data_source_name ,ds.id",projectCode);
		}
		
		for (Map<String,Object> info : connectInfo) {
			String data_source_type = String.valueOf(info.get("data_source_type")).toLowerCase();
			String param = String.valueOf(info.get("param"));
			if (null == param || param.isEmpty()) {
				continue;
			}
			switch (data_source_type) {
			case "mysql":
				String userName = param.split(",")[0];
				if (!userName.equals("root")) {
					break;
				}
				info.put("userName", userName);
				info.put("jdbcUrl", param.split(",")[1]);
				info.put("passWord", param.split(",")[2]);
				info.put("db_type", "mysql");
				break;
				
			case "tidb":
				userName = param.split(",")[0];
				if (!userName.equals("root")) {
					break;
				}
				info.put("userName", userName);
				info.put("jdbcUrl", param.split(",")[1]);
				info.put("passWord", param.split(",")[2]);
				info.put("db_type", "tidb");
				break;
				
			case "hive":
//				analyzeHiveMeta(info); 目前hive是按数仓做收集，不支持物理数据源收集。此处为数据保护伞提前hive元数据使用
				break;
				
			case "oracle":
				info.put("userName", "YJY_SMCS");
				info.put("jdbcUrl", "jdbc:oracle:thin:@172.30.132.157:1522:helowin");
				info.put("passWord", "YJY_SMCS");
				info.put("dsId", "1111111111111");
				analyzeOracleMeta(info);
				break;
				
			case "presto":
//				syncTidbMeta(info);  数据源增加catalog区分
				break;
				
			case "clickhouse":
				userName = param.split(",")[0];
				if (!userName.equals("default")) {
					break;
				}
				info.put("userName", userName);
				info.put("jdbcUrl", param.split(",")[1]);
				info.put("passWord", param.split(",")[2]);
//				syncCKMeta(info);
				break;

			case "elasticsearch":
				syncEsMeta(info);
				break;

			default:
				break;
			}
		}
		return true;
	}
	
	@Transactional
	public boolean analyzeOracleMeta(Map<String,Object> jdbcInfo) {
		String sysUser = "('OLAPSYS','SI_INFORMTN_SCHEMA','OWBSYS','ORDPLUGINS','SPATIAL_WFS_ADMIN_USR','SPATIAL_CSW_ADMIN_USR','XDB','HR','APEX_PUBLIC_USER','OE','DIP','OUTLN','SH','ANONYMOUS','CTXSYS','IX','ORDDATA','MDDATA','PM','BI','OWBSYS_AUDIT','APEX_030200','XS$NULL','APPQOSSYS','ORACLE_OCM','WMSYS','SCOTT','EXFSYS','ORDSYS','MDSYS','FLOWS_FILES','SYSTEM','SYS','MGMT_VIEW','SYSMAN','DBSNMP')";
		Connection conn = null;
		try {
			String userName =  "YJY_SMCS";
			String jdbcUrl = "jdbc:oracle:thin:@172.30.132.157:1522:helowin";
			String passWord = "YJY_SMCS";
			conn = JdbcUtil.getConnection(jdbcUrl, userName, passWord, "oracle");
			String sql = "SELECT 'ANALYZE TABLE '||OWNER||'.'||TABLE_NAME||' COMPUTE STATISTICS' AS SQL FROM DBA_TABLES WHERE OWNER NOT IN " + sysUser;
			List<Map<String, Object>> dataList = JdbcUtil.queryForList(conn, sql);
			for (Map<String, Object> data : dataList) {
				String analyzeSql = String.valueOf(data.get("sql"));
				System.out.println(" >> >> >> analyze oracle " + analyzeSql);
				JdbcUtil.execSql(conn, analyzeSql);
			}
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			return false;
		} finally{
			JdbcUtil.close(conn);
		}
	}
	
	@SuppressWarnings({ "restriction", "unchecked" })
	@Transactional
	public Map<String,Object> syncEsMeta(Map<String,Object> jdbcInfo) {
		Map<String, Object> logMap = new HashMap<>();
		List<MetaTabBean> esAllList = new ArrayList<MetaTabBean>();
		MetaTabBean metaTabBean;
		String userName = String.valueOf(jdbcInfo.get("userName"));
		String url = String.valueOf(jdbcInfo.get("jdbcUrl"));
		String passWord = String.valueOf(jdbcInfo.get("passWord"));
		String dsId = String.valueOf(jdbcInfo.get("dsId"));
		String auth = "";
//		String dsId = "df94e9ea709b470abe9c17ec5113b15b";
//		String userName ="elastic";
//		String url = "http://172.30.132.191:29200";
//		String passWord ="o4DUrtqarWqZg289XHYG";
	 try {
		 if (!CommonUtils.isNull(passWord)) {
			auth = "Basic " + new sun.misc.BASE64Encoder().encode((userName + ":" + AesEncryptUtils.decrypt(passWord)).getBytes());
		 }
		List<String> indexData  =RestTemplateUtil.sendGet(url + "/_cat/indices", auth);
    	List<Map<String, Object>> indexList = new ArrayList<>();
    	List<Map<String, Object>> fieldsList = new ArrayList<>();
    	Map<String, Object> indexMap = null;
    	
    	//存放t_asset_meta_tab_connect使用
    	MetaTabBean mstb = null;
    	List<MetaTabBean> esTabList = new ArrayList<MetaTabBean>();
    	
    	List<String> settingData = null;
    	List<String> mappingData = null;
    	for (String line : indexData) {
    		indexMap = new HashMap<String, Object>();
    		line = line.replaceAll(" +", " ");
    		String[] mes = line.split(" ");
    		String index = mes[2];
    		if (index.startsWith(".")) {
    			continue;
    		}
    		
    		try {
    		settingData = RestTemplateUtil.sendGet(url + "/" + index + "/_settings", auth);
    		JSONObject indices = JSONObject.fromObject(settingData.get(0));
    		Object settings = indices.get(index);
    		JSONObject indexSet = JSONObject.fromObject(settings);
    		Object indexObject = indexSet.get("settings");
    		JSONObject indexJson = JSONObject.fromObject(indexObject);
    		Object ysObject = indexJson.get("index");
    		JSONObject ysSet = JSONObject.fromObject(ysObject);
    		Object number_of_shards = ysSet.get("number_of_shards");
    		Object creation_date = ysSet.get("creation_date");
    		Object number_of_replicas = ysSet.get("number_of_replicas");
    		Object versionObject = ysSet.get("version");
    		Object version = JSONObject.fromObject(versionObject).get("created");
    		creation_date = DateUtils.getDateToString(creation_date);
    		
    		String index_uuid = mes[3];
    		
    		//清理上次同步数据使用
    		metaTabBean = new MetaTabBean();
    		metaTabBean.setTableId(index_uuid);
    		metaTabBean.setTableCode(index);
    		metaTabBean.setJdbcUrl(url);
    		metaTabBean.setDataSourceId(dsId);
    		metaTabBean.setDbType("elasticsearch");
    		esAllList.add(metaTabBean);
    		
    		int store_size = 0;
    		String store = mes[8]==null?"0":mes[8].toLowerCase();
    		if (store.contains("kb")) {
    			store_size = Math.round(Float.parseFloat(store.replace("kb", ""))*1024);
    		} else if (store.contains("mb")) {
    			store_size = Math.round(Float.parseFloat(store.replace("mb", ""))*1024*1024);
    		} else if (store.contains("gb")) {
    			store_size = Math.round(Float.parseFloat(store.replace("gb", ""))*1024*1024*1024);
    		} else if (store.contains("tb")) {
    			store_size = Math.round(Float.parseFloat(store.replace("tb", ""))*1024*1024*1024*1024);
    		} else if (store.contains("pb")) {
    			store_size = Math.round(Float.parseFloat(store.replace("pb", ""))*1024*1024*1024*1024*1024);
    		} else if (store.contains("b")) {
    			store_size = Math.round(Float.parseFloat(store.replace("b", "")));
    		} else {
    			store_size = 0;
    		};
    		
    		indexMap.put("health", mes[0]);
    		indexMap.put("status", mes[1]);
    		indexMap.put("index", index);
    		indexMap.put("uuid", index_uuid);
    		indexMap.put("pri", mes[4]);
    		indexMap.put("rep", mes[5]);
    		indexMap.put("docs_count", mes[6]);
    		indexMap.put("docs_deleted", mes[7]);
    		indexMap.put("store_size", store_size);
    		indexMap.put("pri_store_size", mes[9]);
    		indexMap.put("creation_date", creation_date);
    		indexMap.put("number_of_shards", number_of_shards);
    		indexMap.put("number_of_replicas", number_of_replicas);
    		indexMap.put("version", version);
    		indexMap.put("ds_id", dsId);
    		indexMap.put("setting", String.valueOf(settingData));
    		indexMap.put("mapping", String.valueOf(mappingData));
    		indexList.add(indexMap);
    		
    		//做数据表list使用
    		mstb = new MetaTabBean();
    		mstb.setDbType("elasticsearch");
    		mstb.setDbCode("es");
    		mstb.setTableType(ConstantUtil.INDEX_OBJECT_TYPE);
    		mstb.setTableId(index_uuid);
    		mstb.setTableCode(index);
    		mstb.setTableName(index);
    		mstb.setRowNum(Integer.parseInt(mes[6]));
    		mstb.setRowSize(store_size);
    		mstb.setCreateTableSql(String.valueOf(mappingData));
    		mstb.setJdbcUrl(url);
    		mstb.setJdbcUser(userName);
    		mstb.setJdbcPw(passWord);
    		mstb.setDataSourceId(dsId);
    		mstb.setCreateTime(creation_date);
    		mstb.setLastSyncTime(DateUtils.getCurrentDateTime());
    		esTabList.add(mstb);
    		
    		mappingData = RestTemplateUtil.sendGet(url + "/" + index + "/_mapping", auth);
    		JSONObject indexMappingJson = JSONObject.fromObject(mappingData.get(0));
    		Object indexMappingObje = indexMappingJson.get(index);
//        	Object document = Configuration.defaultConfiguration().jsonProvider().parse(mappingData.get(0));
    		
    		if (Integer.parseInt(version.toString().substring(0, 1)) < 7) {
    			Map<String, Object> type = JsonPath.read(indexMappingObje, "$.mappings");
    			for(Entry<String, Object> properties : type.entrySet()){
    				Map<String, Object> typeMap = (Map<String, Object>) properties.getValue();
    				typeMap.put("type", properties.getKey());
    				esMappingsParse(typeMap,indexMap,fieldsList);
    			}
    		} else {
    			esMappingsParse(JsonPath.read(indexMappingObje, "$.mappings"), indexMap,fieldsList);
    		};

			} catch (Exception e) {
				System.err.println("error index url:"+url + "/" + index + "/_settings");
			}
    	}
		oldDataClean(esAllList);
    	saveEsFieldsMeta(fieldsList);
    	saveEsIndexMeta(indexList);
    	saveTabMeta(esTabList);
//		matchStruct(esAllList);
    	
		logMap.put("db_type", "elasticsearch");
		logMap.put("ds_id", dsId);
		logMap.put("table_num", esAllList.size());
		logMap.put("column_num", fieldsList.size());
		logMap.put("status", "0");
		return logMap;
	 	} catch (Exception e) {
			e.printStackTrace();
			logMap.put("db_type", "elasticsearch");
			logMap.put("ds_id", dsId);
			logMap.put("err_log", e.getMessage());
			logMap.put("status", "1");
			return logMap;
	 	}
	}
	
	@SuppressWarnings("unchecked")
	private List<Map<String, Object>> esMappingsParse(Map<String, Object> typeMap,Map<String, Object> indexMap,List<Map<String, Object>> fieldsList) {
    	Map<String, Object> fieldsMap = null;
    	Map<String, Object> fieldsSubMap = null;
		String index = String.valueOf(indexMap.get("index"));
		String index_uuid = String.valueOf(indexMap.get("uuid"));
		String ds_id = String.valueOf(indexMap.get("ds_id"));
		Map<String, Object> properties = (Map<String, Object>) typeMap.get("properties");
		Object type = typeMap.get("type");
        for(Entry<String, Object> e : properties.entrySet()){
        	String fields = e.getKey();
        	fieldsMap = new HashMap<String,Object>();
        	Map<String, Object> fieldsAttr = (Map<String, Object>) e.getValue();
        	Object fields_type = fieldsAttr.get("type");
        	Object analyzer = fieldsAttr.get("analyzer");
        	Object search_analyzer = fieldsAttr.get("search_analyzer");
        	String fields_index = String.valueOf(fieldsAttr.get("index")==null?"true":fieldsAttr.get("index"));
			Object fields_id = Generators.nameBasedGenerator().generate(index_uuid + fields).toString().replaceAll("-", "");

        	fieldsMap.put("index_uuid", index_uuid);
        	fieldsMap.put("index_name", index);
        	fieldsMap.put("fields_id", fields_id);
        	fieldsMap.put("fields_code", fields);
        	fieldsMap.put("type", type);
        	fieldsMap.put("fields_type", fields_type);
        	fieldsMap.put("analyzer", analyzer);
        	fieldsMap.put("search_analyzer", search_analyzer);
        	fieldsMap.put("fields_index", fields_index);
        	fieldsMap.put("ds_id", ds_id);
        	try {
        		Map<String, Object> subFields = (Map<String, Object>) fieldsAttr.get("fields");
        		if (null != subFields && !subFields.isEmpty()) {
        			for(Entry<String, Object> sub : subFields.entrySet()){
            			Map<String, Object> subFieldsAttr = (Map<String, Object>) sub.getValue();
                    	Object sub_type = subFieldsAttr.get("type");
                    	Object sub_analyzer = subFieldsAttr.get("analyzer");
                    	Object sub_search_analyzer = subFieldsAttr.get("search_analyzer");
                    	String sub_fields_index = String.valueOf(subFieldsAttr.get("index")==null?"true":subFieldsAttr.get("index"));
                    	String sub_fields = sub.getKey();
        				fields_id = Generators.nameBasedGenerator().generate(index_uuid + fields + sub_fields).toString().replaceAll("-", "");
                    	
                    	fieldsSubMap = new HashMap<String,Object>();
                    	fieldsSubMap.put("index_uuid", index_uuid);
                    	fieldsSubMap.put("index_name", index);
                    	fieldsSubMap.put("fields_id", fields_id);
                    	fieldsSubMap.put("fields_code", fields);
                    	fieldsSubMap.put("sub_fields_code", sub_fields);
                    	fieldsSubMap.put("fields_type", sub_type);
                    	fieldsSubMap.put("analyzer", sub_analyzer);
                    	fieldsSubMap.put("search_analyzer", sub_search_analyzer);
                    	fieldsSubMap.put("fields_index", sub_fields_index);
                    	fieldsSubMap.put("ds_id", ds_id);
                    	fieldsList.add(fieldsSubMap);
        			}
        		}
			} catch (Exception e1) {
				e1.printStackTrace();
				}
        	fieldsList.add(fieldsMap);
        	}
		return fieldsList;
	}
	
	@Transactional
	public Map<String,Object> syncOracleMeta(Map<String,Object> jdbcInfo) {
		String sysUser = "('OLAPSYS','SI_INFORMTN_SCHEMA','OWBSYS','ORDPLUGINS','SPATIAL_WFS_ADMIN_USR','SPATIAL_CSW_ADMIN_USR','XDB','HR','APEX_PUBLIC_USER','OE','DIP','OUTLN','SH','ANONYMOUS','CTXSYS','IX','ORDDATA','MDDATA','PM','BI','OWBSYS_AUDIT','APEX_030200','XS$NULL','APPQOSSYS','ORACLE_OCM','WMSYS','SCOTT','EXFSYS','ORDSYS','MDSYS','FLOWS_FILES','SYSTEM','SYS','MGMT_VIEW','SYSMAN','DBSNMP')";
		Connection conn = null;
		List<MetaTabBean> oraAllList = new ArrayList<MetaTabBean>();
		List<MetaColBean> oraColList = new ArrayList<MetaColBean>();
		List<MetaPrtBean> oraPrtList = new ArrayList<MetaPrtBean>();
		Map<String, Object> logMap = new HashMap<>();
		String userName = String.valueOf(jdbcInfo.get("userName"));
		String jdbcUrl = String.valueOf(jdbcInfo.get("jdbcUrl"));
		String passWord = String.valueOf(jdbcInfo.get("passWord"));
		String dsId = String.valueOf(jdbcInfo.get("dsId"));
		String unique = String.valueOf(jdbcInfo.get("unique"));
		try {
			conn = JdbcUtil.getConnection(jdbcUrl, userName, passWord, "oracle");
			String sql = "SELECT t.OWNER,t.TABLE_NAME,c.COMMENTS ,t.NUM_ROWS ,(t.BLOCKS + t.EMPTY_BLOCKS) * 8192 AS table_size ,p.COLUMN_NAME,o.CREATED ,o.LAST_DDL_TIME "
					+ "FROM dba_tab_comments c,DBA_OBJECTS o, DBA_TABLES t LEFT JOIN DBA_PART_KEY_COLUMNS p ON p.OWNER = t.OWNER AND p.name = t.TABLE_NAME "
					+ "WHERE t.OWNER = o.OWNER AND t.TABLE_NAME = o.OBJECT_NAME AND o.OBJECT_TYPE ='TABLE' AND t.OWNER = c.OWNER AND t.TABLE_NAME = c.TABLE_NAME AND t.OWNER  NOT IN " + sysUser;
			List<Map<String, Object>> tabList = JdbcUtil.queryForList(conn, sql);
			
			Map<String,Object> uuidMap = new HashMap<String,Object>();
			MetaTabBean motb;
			for (Map<String,Object> tabDetail : tabList) {
				motb = new MetaTabBean();
				Object db_code = tabDetail.get("owner");
				Object table_code = tabDetail.get("table_name");
				Object uuid = Generators.nameBasedGenerator().generate(unique + db_code + table_code).toString().replaceAll("-", "");
				Object table_name = tabDetail.get("comments");
				Object partition_column = tabDetail.get("column_name");
				long row_num = Long.parseLong(String.valueOf(tabDetail.get("num_rows")==null?"0":tabDetail.get("num_rows")));
//				dba_segments.blocks = dba_tables.blocks + dba_tables.empty_blocks+ 1(SEGMENT HEADER BLOCK)
				long row_size = Long.parseLong(String.valueOf(tabDetail.get("table_size")==null?"0":tabDetail.get("table_size")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(tabDetail.get("created")));
				} catch (Exception e) {
					create_time = null;
				}
				
				Object update_time;
				try {
					update_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(tabDetail.get("last_ddl_time")));
				} catch (Exception e) {
					update_time = null;
				}
				
				motb = new MetaTabBean();
				motb.setDbType("oracle");
				motb.setDbCode(db_code);
				motb.setTableType(ConstantUtil.PHYSICS_OBJECT_TYPE);
				motb.setTableId(uuid);
				motb.setTableCode(table_code);
				motb.setTableName(table_name);
				motb.setCreateUser(db_code.toString().toLowerCase());
				motb.setRowNum(row_num);
				motb.setRowSize(row_size);
				motb.setJdbcUrl(jdbcUrl);
				motb.setJdbcUser(userName);
				motb.setJdbcPw(passWord);
				motb.setDataSourceId(dsId);
				motb.setPartitionColumn(partition_column);
				motb.setCreateTime(create_time);
				motb.setUpdateTime(update_time);
				motb.setLastSyncTime(DateUtils.getCurrentDateTime());
				oraAllList.add(motb);
				uuidMap.put(db_code + ":" + table_code,uuid);
			}
			oldDataClean(oraAllList);
			saveTabMeta(oraAllList);
//			matchStruct(oraAllList);
			
			sql = "select ut.owner,ut.table_name,ucc.column_name as column_name,ucc.column_id ,  do.created ,  "
					+ "case when ucc.data_type = 'char'     then ucc.data_type||'('||ucc.char_length||')'           "
					+ "when ucc.data_type = 'varchar'  then ucc.data_type||'('||ucc.char_length||')'           "
					+ "when ucc.data_type = 'varchar2' then ucc.data_type||'('||ucc.char_length||')'           "
					+ "when ucc.data_type = 'nchar'    then ucc.data_type||'('||ucc.char_length||')'           "
					+ "when ucc.data_type = 'number' then                "
						+ "case when ucc.data_precision is null and ucc.data_scale is null then 'number'                     "
						+ "when ucc.data_precision is null and ucc.data_scale is not null then 'number(38,'||ucc.data_scale||')'                     "
						+ "when ucc.data_scale = 0 then  ucc.data_type||'('||ucc.data_precision||')'                     "
							+ "else ucc.data_type||'('||ucc.data_precision||','||ucc.data_scale||')'                end           "
						+ "when ucc.data_type = 'nvarchar' then ucc.data_type||'('||ucc.char_length||')'           "
						+ "when ucc.data_type = 'nvarchar2' then ucc.data_type||'('||ucc.char_length||')'           "
							+ "else ucc.data_type        end column_type,      "
					+ "acc.comments as column_comments  from dba_tab_columns ucc, dba_tables ut, dba_col_comments acc  ,dba_objects do "
					+ "where ut.owner = do.owner and ut.table_name = do.object_name and do.object_type = 'TABLE'  and ucc.owner  = ut.owner    and ucc.table_name  = ut.table_name    and ut.tablespace_name is not null       and acc.owner       = ucc.owner    and acc.table_name  = ucc.table_name    and acc.column_name = ucc.column_name  "
					+ "and ucc.owner NOT IN " + sysUser;
			List<Map<String, Object>> colList = JdbcUtil.queryForList(conn, sql);
			
			MetaColBean mccb;
			for (Map<String,Object> colDetail : colList) {
				mccb = new MetaColBean();
				Object db_code = colDetail.get("owner");
				Object table_code = colDetail.get("table_name");
				Object table_id = uuidMap.get(db_code + ":" + table_code);
				Object column_code = colDetail.get("column_name");
				Object column_id = Generators.nameBasedGenerator().generate(String.valueOf(table_id) + String.valueOf(column_code)).toString().replaceAll("-", "");
				Object column_name = colDetail.get("column_comments");
				Object column_type = colDetail.get("column_type");
				int position = Integer.parseInt(String.valueOf(colDetail.get("column_id")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(colDetail.get("created")));
				} catch (Exception e) {
					create_time = null;
				}
				mccb.setDataSourceId(dsId);
				mccb.setDbType("oracle");
				mccb.setDbCode(db_code);
				mccb.setTableId(table_id);
				mccb.setTableCode(table_code);
				mccb.setColumnId(column_id);
				mccb.setColumnCode(column_code);
				mccb.setColumnName(column_name);
				mccb.setColumnType(column_type);
				mccb.setPosition(position);
				mccb.setCreateTime(create_time);
				mccb.setLastSyncTime(DateUtils.getCurrentDateTime());
				oraColList.add(mccb);
			}
			saveColMeta(oraColList);
			
			sql = "SELECT p.TABLE_OWNER ,p.table_name,c.COLUMN_NAME ,p.partition_name, p.high_value,p.NUM_ROWS ,(p.BLOCKS + p.EMPTY_BLOCKS) * 8192 AS partition_size,o.CREATED "
					+ "FROM DBA_TAB_PARTITIONS p,DBA_PART_KEY_COLUMNS c, DBA_OBJECTS o "
					+ "WHERE c.owner = p.TABLE_OWNER AND c.name = p.table_name AND p.TABLE_OWNER = o.OWNER AND p.table_name = o.OBJECT_NAME AND o.OBJECT_TYPE ='TABLE PARTITION' AND p.TABLE_OWNER  NOT IN " + sysUser;
			List<Map<String, Object>> prtList = JdbcUtil.queryForList(conn, sql);
			
			MetaPrtBean mopb;
			for (Map<String,Object> parDetail : prtList) {
				mopb = new MetaPrtBean();
				Object db_code = parDetail.get("table_owner");
				Object table_code = parDetail.get("table_name");
				Object table_id = uuidMap.get(db_code + ":" + table_code);
				Object partition_code = parDetail.get("column_name");

				Object partition_name = parDetail.get("partition_name");
				Object partition_value = parDetail.get("high_value");
				partition_value = partition_name + ":[" + partition_value + "]";
				Object partition_id = Generators.nameBasedGenerator().generate(String.valueOf(table_id) + String.valueOf(partition_value)).toString().replaceAll("-", "");
				long row_num = Long.parseLong(String.valueOf(parDetail.get("num_rows")==null?"0":parDetail.get("num_rows")));
				long row_size = Long.parseLong(String.valueOf(parDetail.get("partition_size")==null?"0":parDetail.get("partition_size")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(parDetail.get("created")));
				} catch (Exception e) {
					create_time = null;
				}
				mopb.setDataSourceId(dsId);
				mopb.setDbType("oracle");
				mopb.setDbCode(db_code);
				mopb.setTableId(table_id+"");
				mopb.setTableCode(table_code);
				mopb.setPartition_id(partition_id+"");
				mopb.setPartition_code(partition_code);
				mopb.setPartition_value(partition_value);
				mopb.setRowNum(row_num);
				mopb.setRowSize(row_size);
				mopb.setCreateTime(create_time);
				mopb.setLastSyncTime(DateUtils.getCurrentDateTime());
				oraPrtList.add(mopb);
			}
			saveParMeta(oraPrtList);
			
			logMap.put("db_type", "oracle");
			logMap.put("ds_id", dsId);
			logMap.put("table_num", oraAllList.size());
			logMap.put("column_num", oraColList.size());
			logMap.put("partition_num", oraPrtList.size());
			logMap.put("status", "0");
			return logMap;
		} catch (Exception e) {
			e.printStackTrace();
			logMap.put("db_type", "oracle");
			logMap.put("ds_id", dsId);
			logMap.put("err_log", e.getMessage());
			logMap.put("status", "1");
			return logMap;
		} finally {
			JdbcUtil.close(conn);
		}
	}
	
	@Transactional
	public Map<String,Object> syncCKMeta(Map<String,Object> jdbcInfo) {
		Connection conn = null;
		List<MetaTabBean> ckAllList = new ArrayList<MetaTabBean>();
		List<MetaColBean> ckColList = new ArrayList<MetaColBean>();
		List<MetaPrtBean> ckPrtList = new ArrayList<MetaPrtBean>();
		Map<String, Object> logMap = new HashMap<>();
		String userName = String.valueOf(jdbcInfo.get("userName"));
		String jdbcUrl = String.valueOf(jdbcInfo.get("jdbcUrl"));
		String passWord = String.valueOf(jdbcInfo.get("passWord"));
		String dsId = String.valueOf(jdbcInfo.get("dsId"));
		try {
			String unique = JdbcUtil.parseJdbc(jdbcUrl, jdbcInfo.get("data_source_type").toString()).get("unique");
			conn = JdbcUtil.getConnection(jdbcUrl, userName, AesEncryptUtils.decrypt(passWord), "ck");
			List<Map<String, Object>> tabList = JdbcUtil.queryForList(conn, "select version() as version");
			String version = String.valueOf(tabList.get(0).get("version")).substring(0,2);
			try {
				if (Integer.parseInt(version) < 20) {
					//2.0以下版本，暂不支持
					logMap.put("db_type", "clickhouse");
					logMap.put("ds_id", dsId);
					logMap.put("err_log", version + " 20版本以下不支持");
					logMap.put("status", "1");
					return logMap;
					}
			} catch (Exception e) {
				logMap.put("db_type", "clickhouse");
				logMap.put("ds_id", dsId);
				logMap.put("err_log", e.getMessage());
				logMap.put("status", "1");
				return logMap;
			}
			String sql = "select database,uuid,name,comment,max(total_rows) as total_rows,max(total_bytes) as total_bytes,max(create_table_query) as create_table_query ,partition_key,max(metadata_modification_time) as metadata_modification_time,MAX(user_name) as user_name from ( "
					+ "select t.database,t.uuid,t.name,'' as comment,t.total_rows,t.total_bytes,t.create_table_query ,t.partition_key,t.metadata_modification_time ,'' as user_name  from system.tables t "
					+ "where database !='system' "
					+ "union all  "
					+ "select t.database,t.uuid,t.name,'' as comment,t.total_rows,t.total_bytes,t.create_table_query ,t.partition_key,t.metadata_modification_time ,'' as user_name  from system.tables t "
					+ "where database !='system' ) t  "
					+ "group by database,uuid,name,comment,partition_key ";
			tabList = JdbcUtil.queryForList(conn, sql);
			
			Map<String,Object> uuidMap = new HashMap<String,Object>();
			MetaTabBean mctb;
			for (Map<String,Object> tabDetail : tabList) {
				mctb = new MetaTabBean();
				Object db_code = tabDetail.get("database");
				Object table_code = tabDetail.get("name");
				Object id = Generators.nameBasedGenerator().generate(unique + db_code + table_code).toString().replaceAll("-", "");
//				Object uuid = tabDetail.get("uuid");
				Object create_table_sql = tabDetail.get("create_table_query");
				Object partition_column = tabDetail.get("partition_key");
				Object user_name = tabDetail.get("user_name")==null? "default":tabDetail.get("user_name");
				long row_num = Long.parseLong(String.valueOf(tabDetail.get("total_rows")==null?"0":tabDetail.get("total_rows")));
				long row_size = Long.parseLong(String.valueOf(tabDetail.get("total_bytes")==null?"0":tabDetail.get("total_bytes")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(tabDetail.get("metadata_modification_time")));
				} catch (Exception e) {
					create_time = null;
				}
				Object table_name = String.valueOf(tabDetail.get("comment"));
				mctb = new MetaTabBean();
				mctb.setDbType("clickhouse");
				mctb.setDbCode(db_code);
				mctb.setTableType(ConstantUtil.PHYSICS_OBJECT_TYPE);
				mctb.setTableId(id+"");
				mctb.setTableCode(table_code);
				mctb.setTableName(table_name);
				mctb.setCreateUser(user_name);
				mctb.setRowNum(row_num);
				mctb.setRowSize(row_size);
				mctb.setJdbcUrl(jdbcUrl);
				mctb.setJdbcUser(userName);
				mctb.setJdbcPw(passWord);
				mctb.setDataSourceId(dsId);
				mctb.setCreateTableSql(create_table_sql);
				mctb.setPartitionColumn(partition_column);
				mctb.setCreateTime(create_time);
				mctb.setUpdateTime(create_time);
				mctb.setLastSyncTime(DateUtils.getCurrentDateTime());
				ckAllList.add(mctb);
				uuidMap.put(db_code + ":" + table_code,id+"");
			}
			oldDataClean(ckAllList);
			saveTabMeta(ckAllList);
//			matchStruct(ckAllList);
			
			sql = "select c.database ,t.uuid ,c.table,c.name,c.comment ,c.type ,c.position,t.metadata_modification_time from system.columns c,system.tables t  where c.database !='system' and c.database = t.database and c.table = t.name ";
			List<Map<String, Object>> colList = JdbcUtil.queryForList(conn, sql);
			
			MetaColBean mccb;
			for (Map<String,Object> colDetail : colList) {
				mccb = new MetaColBean();
				Object db_code = colDetail.get("database");
				Object table_code = colDetail.get("table");
				Object table_id = Generators.nameBasedGenerator().generate(unique + db_code + table_code).toString().replaceAll("-", "");
				Object column_code = colDetail.get("name");
				Object column_id = Generators.nameBasedGenerator().generate(String.valueOf(table_id) + String.valueOf(column_code)).toString().replaceAll("-", "");
				Object column_name = String.valueOf(colDetail.get("comment"));
				Object column_type = String.valueOf(colDetail.get("type"));
				int position = Integer.parseInt(String.valueOf(colDetail.get("position")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(colDetail.get("metadata_modification_time")));
				} catch (Exception e) {
					create_time = null;
				}
				mccb.setDataSourceId(dsId);
				mccb.setDbType("clickhouse");
				mccb.setDbCode(db_code);
				mccb.setTableId(table_id+"");
				mccb.setTableCode(table_code);
				mccb.setColumnId(column_id+"");
				mccb.setColumnCode(column_code);
				mccb.setColumnName(column_name);
				mccb.setColumnType(column_type);
				mccb.setPosition(position);
				mccb.setCreateTime(create_time);
				mccb.setLastSyncTime(DateUtils.getCurrentDateTime());
				ckColList.add(mccb);
			}
			saveColMeta(ckColList);
			
			sql = "select p.database,t.uuid as table_id,p.table,p.partition_id,t.partition_key as partition_code,p.partition as partition_value,p.rows,p.bytes_on_disk,modification_time "
					+ "from system.parts p,system.tables t "
					+ "where p.database != 'system' and p.database = t.database and p.table = t.name ";
			List<Map<String, Object>> prtList = JdbcUtil.queryForList(conn, sql);
			
			MetaPrtBean mcpb;
			for (Map<String,Object> parDetail : prtList) {
				mcpb = new MetaPrtBean();
				Object db_code = parDetail.get("database");
				Object table_code = parDetail.get("table");
				Object table_id = Generators.nameBasedGenerator().generate(unique + db_code + table_code).toString().replaceAll("-", "");
				Object partition_id = parDetail.get("partition_id");
				Object partition_code = parDetail.get("partition_code");
				Object partition_value = parDetail.get("partition_value");
				long row_num = Long.parseLong(String.valueOf(parDetail.get("rows")==null?"0":parDetail.get("rows")));
				long row_size = Long.parseLong(String.valueOf(parDetail.get("bytes_on_disk")==null?"0":parDetail.get("bytes_on_disk")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(parDetail.get("modification_time")));
				} catch (Exception e) {
					create_time = null;
				}
				mcpb.setDataSourceId(dsId);
				mcpb.setDbType("clickhouse");
				mcpb.setDbCode(db_code);
				mcpb.setTableId(table_id);
				mcpb.setTableCode(table_code);
				mcpb.setPartition_id(partition_id);
				mcpb.setPartition_code(partition_code);
				mcpb.setPartition_value(partition_value);
				mcpb.setRowNum(row_num);
				mcpb.setRowSize(row_size);
				mcpb.setCreateTime(create_time);
				mcpb.setLastSyncTime(DateUtils.getCurrentDateTime());
				ckPrtList.add(mcpb);
			}
			saveParMeta(ckPrtList);
			
			logMap.put("db_type", "clickhouse");
			logMap.put("ds_id", dsId);
			logMap.put("table_num", ckAllList.size());
			logMap.put("column_num", ckColList.size());
			logMap.put("partition_num", ckPrtList.size());
			logMap.put("status", "0");
		return logMap;
		} catch (Exception e) {
			e.printStackTrace();
			logMap.put("db_type", "clickhouse");
			logMap.put("ds_id", dsId);
			logMap.put("err_log", e.getMessage());
			logMap.put("status", "1");
			return logMap;
		} finally {
			JdbcUtil.close(conn);
		}
	}
	
	/**
	 * 同步tidb表、列、分区
	 * @return 
	 * 
	 * @throws Exception
	 */
	@Transactional
	public Map<String,Object> syncTidbMeta(Map<String,Object> jdbcInfo) {
		Connection conn = null;
		List<MetaTabBean> tidbAllList = new ArrayList<MetaTabBean>();
		List<MetaColBean> tidbcolList = new ArrayList<MetaColBean>();
		List<MetaPrtBean> tidbPrtList = new ArrayList<MetaPrtBean>();
		Map<String, Object> logMap = new HashMap<>();
		String userName = String.valueOf(jdbcInfo.get("userName"));
		String jdbcUrl = String.valueOf(jdbcInfo.get("jdbcUrl"));
		String passWord = String.valueOf(jdbcInfo.get("passWord"));
		String dsId = String.valueOf(jdbcInfo.get("dsId"));
		String db_type = String.valueOf(jdbcInfo.get("db_type"));
		try {
			String unique = JdbcUtil.parseJdbc(jdbcUrl, jdbcInfo.get("data_source_type").toString()).get("unique");
			conn = JdbcUtil.getConnection(jdbcUrl, userName, AesEncryptUtils.decrypt(passWord), "mysql");
			String sql = "select t.TABLE_SCHEMA,t.TABLE_NAME,t.TABLE_ROWS,t.DATA_LENGTH, t.CREATE_TIME ,t.UPDATE_TIME, t.TABLE_COMMENT,p.PARTITION_EXPRESSION,d.user from information_schema.TABLES t "
					+ "left join (select distinct TABLE_SCHEMA,TABLE_NAME,PARTITION_EXPRESSION from information_schema.PARTITIONS where PARTITION_NAME is not null) p "
					+ "on t.TABLE_SCHEMA = p.TABLE_SCHEMA  and t.TABLE_NAME = p.TABLE_NAME left join (select db,group_concat(user) user from mysql.db group by db) d on t.TABLE_SCHEMA = d.DB "
					+ "where t.table_type='BASE TABLE' ";
			
			List<Map<String, Object>> tabList = JdbcUtil.queryForList(conn, sql);
			Map<String,Object> uuidMap = new HashMap<String,Object>();
			MetaTabBean mttb;
			for (Map<String,Object> tabDetail : tabList) {
				mttb = new MetaTabBean();
				Object db_code = tabDetail.get("table_schema");
				Object table_code = tabDetail.get("table_name");
				Object id = Generators.nameBasedGenerator().generate(unique + db_code + table_code).toString().replaceAll("-", "");
				long row_num = Long.parseLong(String.valueOf(tabDetail.get("table_rows")==null?"0":tabDetail.get("table_rows")));
				long row_size = Long.parseLong(String.valueOf(tabDetail.get("data_length")==null?"0":tabDetail.get("data_length")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(tabDetail.get("create_time")));
				} catch (Exception e) {
					create_time = null;
				}
				Object update_time;
				try {
					update_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(tabDetail.get("update_time")));
				} catch (Exception e) {
					update_time = null;
				}
				Object table_name = String.valueOf(tabDetail.get("table_comment"));
				Object partition_code = tabDetail.get("partition_expression");

				try {
					int start = StringUtils.ordinalIndexOf(String.valueOf(partition_code), "`", 1)+1;
					int end = StringUtils.ordinalIndexOf(String.valueOf(partition_code), "`", 2);
					partition_code = String.valueOf(partition_code).substring(start, end);
				} catch (Exception e) {
				}
				
				Object user = tabDetail.get("user")==null? "root":tabDetail.get("user");
				
				mttb = new MetaTabBean();
				mttb.setDbType(db_type);
				mttb.setDbCode(db_code);
				mttb.setTableType(ConstantUtil.PHYSICS_OBJECT_TYPE);
				mttb.setTableId(id+"");
				mttb.setTableCode(table_code);
				mttb.setTableName(table_name);
				mttb.setRowNum(row_num);
				mttb.setRowSize(row_size);
				mttb.setCreateUser(user);
				mttb.setJdbcUrl(jdbcUrl);
				mttb.setJdbcUser(userName);
				mttb.setJdbcPw(passWord);
				mttb.setDataSourceId(dsId);
				mttb.setCreateTime(create_time);
				mttb.setUpdateTime(update_time);
				mttb.setPartitionColumn(partition_code);
				mttb.setLastSyncTime(DateUtils.getCurrentDateTime());
				tidbAllList.add(mttb);
				uuidMap.put(db_code + ":" + table_code,id);
			}
			oldDataClean(tidbAllList);
			saveTabMeta(tidbAllList);
//			matchStruct(tidbAllList);
			
			sql = "SELECT c.TABLE_SCHEMA,c.TABLE_NAME,c.COLUMN_NAME,c.COLUMN_COMMENT ,c.COLUMN_TYPE,t.CREATE_TIME,c.ORDINAL_POSITION  from information_schema.COLUMNS c,information_schema.TABLES t where t.table_type='BASE TABLE' and t.TABLE_SCHEMA = c.TABLE_SCHEMA and t.TABLE_NAME = c.TABLE_NAME";
			List<Map<String, Object>> colList = JdbcUtil.queryForList(conn, sql);
			
			MetaColBean mtcb;
			for (Map<String,Object> colDetail : colList) {
				mtcb = new MetaColBean();
				Object db_code = colDetail.get("table_schema");
				Object table_code = colDetail.get("table_name");
				Object table_id = uuidMap.get(db_code + ":" + table_code);
				Object column_code = colDetail.get("column_name");
				Object column_id = Generators.nameBasedGenerator().generate(String.valueOf(table_id) + String.valueOf(column_code)).toString().replaceAll("-", "");
				Object column_name = colDetail.get("column_comment");
				Object column_type = colDetail.get("column_type");
				int ordinal_position = Integer.parseInt(String.valueOf(colDetail.get("ordinal_position")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(colDetail.get("create_time")));
				} catch (Exception e) {
					create_time = null;
				}
				mtcb.setDataSourceId(dsId);
				mtcb.setDbType(db_type);
				mtcb.setDbCode(db_code);
				mtcb.setTableId(table_id+"");
				mtcb.setTableCode(table_code);
				mtcb.setColumnId(column_id+"");
				mtcb.setColumnCode(column_code);
				mtcb.setColumnName(column_name);
				mtcb.setColumnType(column_type);
				mtcb.setCreateTime(create_time);
				mtcb.setPosition(ordinal_position);
				mtcb.setLastSyncTime(DateUtils.getCurrentDateTime());
				tidbcolList.add(mtcb);
			}
			saveColMeta(tidbcolList);
			
			sql = "select p.TABLE_SCHEMA,p.TABLE_NAME,p.PARTITION_NAME,p.PARTITION_EXPRESSION ,p.TABLE_ROWS,p.DATA_LENGTH, p.CREATE_TIME "
					+ "from information_schema.PARTITIONS p ,information_schema.TABLES t "
					+ "where t.table_type='BASE TABLE' and t.CREATE_OPTIONS = 'partitioned' and t.TABLE_SCHEMA = p.TABLE_SCHEMA and t.TABLE_NAME = p.TABLE_NAME";
			List<Map<String, Object>> prtList = JdbcUtil.queryForList(conn, sql);
			
			MetaPrtBean mtpb;
			for (Map<String,Object> parDetail : prtList) {
				mtpb = new MetaPrtBean();
				Object db_code = parDetail.get("table_schema");
				Object table_code = parDetail.get("table_name");
				Object table_id = uuidMap.get(db_code + ":" + table_code);
				Object partition_code = parDetail.get("partition_expression");

				try { 
					int start = StringUtils.ordinalIndexOf(String.valueOf(partition_code), "`", 1)+1;
					int end = StringUtils.ordinalIndexOf(String.valueOf(partition_code), "`", 2);
					partition_code = String.valueOf(partition_code).substring(start, end);
				} catch (Exception e) {
				}
				
				Object partition_name = parDetail.get("partition_name");
				Object partition_id = Generators.nameBasedGenerator().generate(String.valueOf(table_id) + String.valueOf(partition_name)).toString().replaceAll("-", "");
				long row_num = Long.parseLong(String.valueOf(parDetail.get("table_rows")==null?"0":parDetail.get("table_rows")));
				long row_size = Long.parseLong(String.valueOf(parDetail.get("data_length")==null?"0":parDetail.get("data_length")));
				Object create_time;
				try {
					create_time = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parseObject(String.valueOf(parDetail.get("create_time")));
				} catch (Exception e) {
					create_time = null;
				}
				mtpb.setDataSourceId(dsId);
				mtpb.setDbType(db_type);
				mtpb.setDbCode(db_code);
				mtpb.setTableId(table_id+"");
				mtpb.setTableCode(table_code);
				mtpb.setPartition_id(partition_id+"");
				mtpb.setPartition_code(partition_code);
				mtpb.setPartition_value(partition_name);
				mtpb.setRowNum(row_num);
				mtpb.setRowSize(row_size);
				mtpb.setCreateTime(create_time);
				mtpb.setLastSyncTime(DateUtils.getCurrentDateTime());
				tidbPrtList.add(mtpb);
			}
			saveParMeta(tidbPrtList);
			
			logMap.put("db_type", db_type);
			logMap.put("ds_id", dsId);
			logMap.put("table_num", tidbAllList.size());
			logMap.put("column_num", tidbcolList.size());
			logMap.put("partition_num", tidbPrtList.size());
			logMap.put("status", "0");
		return logMap;
		} catch (Exception e) {
//			e.printStackTrace();
			logMap.put("db_type", db_type);
			logMap.put("ds_id", dsId);
			logMap.put("err_log", e.getMessage());
			logMap.put("status", "1");
			return logMap;
		} finally {
			JdbcUtil.close(conn);
		}
	}
	
	private void oldDataClean(List<MetaTabBean> allList) {
		Map<Object, Object> tabMap = new HashMap<>();
		Object dsId = allList.get(0).getDataSourceId();
		List<Map<String, Object>> trailData = gJdbc.queryForList("select object_code from t_asset_m_trail where ds_id = ? and source_type !='hive'",dsId);
		for (MetaTabBean tabBean : allList) {
			Object tableCode = tabBean.getTableCode();
			tabMap.put(tableCode, "");
		}
		
		StringBuffer trailBf = new StringBuffer();
		for (Map<String, Object> trail : trailData) {
			String object_code = String.valueOf(trail.get("object_code"));
			if (!tabMap.containsKey(object_code)) {
				trailBf.append("'").append(object_code).append("',");
			}
		}
		trailBf = trailBf.length()>0? trailBf.deleteCharAt(trailBf.length()-1):trailBf;
		if (trailBf.length() > 0) {
			gJdbc.update("delete from t_asset_m_trail where ds_id = ? and source_type !='hive' and object_code in (" + trailBf.toString() + ")",dsId);
		}
//		gJdbc.update("delete p from t_asset_meta_struct p where exists (select 1 from t_asset_meta_tab_collect t where t.table_id = p.table_id and t.ds_id =?)",dsId);
//		gJdbc.update("delete p from t_asset_meta_prt_collect p where exists (select 1 from t_asset_meta_tab_collect t where t.table_id = p.table_id and t.ds_id =?)",dsId);
//		gJdbc.update("delete p from t_asset_meta_col_collect p where exists (select 1 from t_asset_meta_tab_collect t where t.table_id = p.table_id and t.ds_id =?)",dsId);
//		gJdbc.update("delete p from t_asset_es_fields_collect p where exists (select 1 from t_asset_es_index_collect t where t.id = p.index_id and t.ds_id =?)",dsId);
		
		gJdbc.update("delete from t_asset_meta_prt_collect where ds_id =? ",dsId);
		gJdbc.update("delete from t_asset_meta_col_collect where ds_id =? ",dsId);
		gJdbc.update("delete from t_asset_meta_tab_collect where ds_id =? ",dsId);
		gJdbc.update("delete from t_asset_es_fields_collect where ds_id =? ",dsId);
		gJdbc.update("delete from t_asset_es_index_collect where ds_id =? ",dsId);
	}
	
	private boolean saveTabMeta(final List<MetaTabBean> dbResult) throws Exception {
		try {
			String sql = "replace into t_asset_meta_tab_collect(db_type,db_code,table_id,table_code,table_name,row_num,raw_size,create_table_sql,partition_column,jdbc_url,jdbc_user,jdbc_pw,ds_id,create_time,create_user,update_time,sync_time,object_type) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).getDbType());
					ps.setObject(2, dbResult.get(i).getDbCode());
					ps.setObject(3, String.valueOf(dbResult.get(i).getTableId()));
					ps.setObject(4, dbResult.get(i).getTableCode());
					ps.setObject(5, dbResult.get(i).getTableName());
					ps.setObject(6, dbResult.get(i).getRowNum());
					ps.setObject(7, dbResult.get(i).getRowSize());
					ps.setObject(8, dbResult.get(i).getCreateTableSql());
					ps.setObject(9, dbResult.get(i).getPartitionColumn());
					ps.setObject(10, dbResult.get(i).getJdbcUrl());
					ps.setObject(11, dbResult.get(i).getJdbcUser());
					ps.setObject(12, dbResult.get(i).getJdbcPw());
					ps.setObject(13, dbResult.get(i).getDataSourceId());
					ps.setObject(14, dbResult.get(i).getCreateTime());
					ps.setObject(15, dbResult.get(i).getCreateUser());
					ps.setObject(16, dbResult.get(i).getUpdateTime());
					ps.setObject(17, dbResult.get(i).getLastSyncTime());
					ps.setObject(18, dbResult.get(i).getTableType());
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			throw new Exception(e.getMessage());
		}
	}
	
	private boolean saveColMeta(final List<MetaColBean> dbResult) throws Exception {
		try {
			String sql = "replace into t_asset_meta_col_collect(db_code,table_id,table_code,column_id,column_code,column_name,column_type,position,create_time,sync_time,db_type,ds_id) values(?,?,?,?,?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).getDbCode());
					ps.setObject(2, String.valueOf(dbResult.get(i).getTableId()));
					ps.setObject(3, dbResult.get(i).getTableCode());
					ps.setObject(4, dbResult.get(i).getColumnId());
					ps.setObject(5, dbResult.get(i).getColumnCode());
					ps.setObject(6, dbResult.get(i).getColumnName());
					ps.setObject(7, dbResult.get(i).getColumnType());
					ps.setObject(8, dbResult.get(i).getPosition());
					ps.setObject(9, dbResult.get(i).getCreateTime());
					ps.setObject(10, dbResult.get(i).getLastSyncTime());
					ps.setObject(11, dbResult.get(i).getDbType());
					ps.setObject(12, dbResult.get(i).getDataSourceId());
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			throw new Exception(e.getMessage());
		}
	}
	
	private boolean saveParMeta(final List<MetaPrtBean> dbResult) throws Exception {
		try {
			String sql = "replace into t_asset_meta_prt_collect(db_code,table_id,table_code,partition_id,partition_code,partition_value,row_num,raw_size,create_time,sync_time,db_type,ds_id) values(?,?,?,?,?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).getDbCode());
					ps.setObject(2, String.valueOf(dbResult.get(i).getTableId()));
					ps.setObject(3, dbResult.get(i).getTableCode());
					ps.setObject(4, dbResult.get(i).getPartition_id());
					ps.setObject(5, dbResult.get(i).getPartition_code());
					ps.setObject(6, dbResult.get(i).getPartition_value());
					ps.setObject(7, dbResult.get(i).getRowNum());
					ps.setObject(8, dbResult.get(i).getRowSize());
					ps.setObject(9, dbResult.get(i).getCreateTime());
					ps.setObject(10, dbResult.get(i).getLastSyncTime());
					ps.setObject(11, dbResult.get(i).getDbType());
					ps.setObject(12, dbResult.get(i).getDataSourceId());
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			throw new Exception(e.getMessage());
		}
	}
	
	private boolean saveStructMeta(final List<Map<String,Object>> dbResult) throws Exception {
		try {
			String sql = "replace into t_asset_meta_struct(busin_id,busin_code,busin_name,project_id,project_code,project_name,table_id,table_code,db_type,ds_id,db_code) values(?,?,?,?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).get("busin_id"));
					ps.setObject(2, dbResult.get(i).get("busin_code"));
					ps.setObject(3, dbResult.get(i).get("busin_name"));
					ps.setObject(4, dbResult.get(i).get("project_id"));
					ps.setObject(5, dbResult.get(i).get("project_code"));
					ps.setObject(6, dbResult.get(i).get("project_name"));
					ps.setObject(7, String.valueOf(dbResult.get(i).get("table_id")));
					ps.setObject(8, dbResult.get(i).get("table_code"));
					ps.setObject(9, dbResult.get(i).get("db_type"));
					ps.setObject(10, dbResult.get(i).get("ds_id"));
					ps.setObject(11, dbResult.get(i).get("db_code"));
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			throw new Exception(e.getMessage());
		}
	}
	
	private boolean saveEsIndexMeta(final List<Map<String,Object>> dbResult) {
		try {
			String sql = "insert into t_asset_es_index_collect(id,index_code,health,status,pri,rep,docs_count,docs_deleted,store_size,pri_store_size,setting,mapping,ds_id,sync_time,create_time,number_of_shards,number_of_replicas,version) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).get("uuid"));
					ps.setObject(2, dbResult.get(i).get("index"));
					ps.setObject(3, dbResult.get(i).get("health"));
					ps.setObject(4, dbResult.get(i).get("status"));
					ps.setObject(5, dbResult.get(i).get("pri"));
					ps.setObject(6, dbResult.get(i).get("rep"));
					ps.setObject(7, dbResult.get(i).get("docs_count"));
					ps.setObject(8, dbResult.get(i).get("docs_deleted"));
					ps.setObject(9, dbResult.get(i).get("store_size"));
					ps.setObject(10, dbResult.get(i).get("pri_store_size"));
					ps.setObject(11, dbResult.get(i).get("setting"));
					ps.setObject(12, dbResult.get(i).get("mapping"));
					ps.setObject(13, dbResult.get(i).get("ds_id"));
					ps.setObject(14, DateUtils.getCurrentDateTime());
					
					ps.setObject(15, dbResult.get(i).get("creation_date"));
					ps.setObject(16, dbResult.get(i).get("number_of_shards"));
					ps.setObject(17, dbResult.get(i).get("number_of_replicas"));
					ps.setObject(18, dbResult.get(i).get("version"));
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			return false;
		}
	}
	
	private boolean saveEsFieldsMeta(final List<Map<String,Object>> dbResult) {
		try {
			String sql = "insert into t_asset_es_fields_collect(index_id,index_code,fields_id,fields_code,sub_fields_code,type,analyzer,search_analyzer,fields_index,sync_time,ds_id,fields_type) values(?,?,?,?,?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).get("index_uuid"));
					ps.setObject(2, dbResult.get(i).get("index_name"));
					ps.setObject(3, dbResult.get(i).get("fields_id"));
					ps.setObject(4, dbResult.get(i).get("fields_code"));
					ps.setObject(5, dbResult.get(i).get("sub_fields_code"));
					ps.setObject(6, dbResult.get(i).get("type"));
					ps.setObject(7, dbResult.get(i).get("analyzer"));
					ps.setObject(8, dbResult.get(i).get("search_analyzer"));
					ps.setObject(9, dbResult.get(i).get("fields_index"));
					ps.setObject(10, DateUtils.getCurrentDateTime());
					ps.setObject(11, dbResult.get(i).get("ds_id"));
					ps.setObject(12, dbResult.get(i).get("fields_type"));
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			return false;
		}
	}
	
	private boolean saveLog(final List<Map<String,Object>> dbResult) {
		try {
			String sql = "insert into t_asset_meta_sync_log(db_type,ds_id,table_num,column_num,partition_num,err_log,status,sync_time) values(?,?,?,?,?,?,?,?)";
			gJdbc.batchUpdate(sql, new BatchPreparedStatementSetter() {
				@Override
				public void setValues(PreparedStatement ps, int i) throws SQLException {
					ps.setObject(1, dbResult.get(i).get("db_type"));
					ps.setObject(2, dbResult.get(i).get("ds_id"));
					ps.setObject(3, dbResult.get(i).get("table_num"));
					ps.setObject(4, dbResult.get(i).get("column_num"));
					ps.setObject(5, dbResult.get(i).get("partition_num"));
					ps.setObject(6, dbResult.get(i).get("err_log"));
					ps.setObject(7, dbResult.get(i).get("status"));
					ps.setObject(8, DateUtils.getCurrentDateTime());
				}

				@Override
				public int getBatchSize() {
					return dbResult.size();
				}
			});
			return true;
		} catch (Exception e) {
			e.printStackTrace();
			return false;
		}
	}
	
	/**
	 * 匹配tidb表，对应的项目、业务板块
	 * 数据源中存在root权限数据源，但项目关联数据源没有root权限，此处没有做关联
	 * @param tabList
	 * @throws Exception
	 */
	public void matchStructOld(List<MetaTabBean> allList) throws Exception {
		Map<String, Object> structMap = new HashMap<>();
		List<Map<String, Object>> structData = new ArrayList<>();
		List<Map<String, Object>> dsData = gJdbc.queryForList("select bf.busin_id,bf.busin_code ,bf.busin_name ,p.id project_id,p.project_code,p.project_name "
				+ "from t_meta_business_field bf ,t_meta_project_ds ds ,t_meta_project p "
				+ "where bf.busin_id = p.busin_id  and ds.project_id = p.id and ds.ds_id = ?",allList.get(0).getDataSourceId());
		
		for (Map<String, Object> ds : dsData) {
				String busin_id = String.valueOf(ds.get("busin_id"));
				String busin_code = String.valueOf(ds.get("busin_code"));
				String busin_name = String.valueOf(ds.get("busin_name"));
				String project_id = String.valueOf(ds.get("project_id"));
				String project_code = String.valueOf(ds.get("project_code"));
				String project_name = String.valueOf(ds.get("project_name"));
				
				for (MetaTabBean tab : allList) {
					structMap = new HashMap<String, Object>();
					structMap.put("busin_id", busin_id);
					structMap.put("busin_code", busin_code);
					structMap.put("busin_name", busin_name);
					structMap.put("project_id", project_id);
					structMap.put("project_code", project_code);
					structMap.put("project_name", project_name);
					structMap.put("table_id", tab.getTableId());
					structMap.put("table_code", tab.getTableCode());
					structMap.put("db_type", tab.getDbType());
					structMap.put("ds_id", tab.getDataSourceId());
					structData.add(structMap);
				}
		}
//		saveStructMeta(structData);
	}
	
	private void matchStruct(List<Map<String, Object>> struct) throws Exception {
		List<Map<String, Object>> structData = new ArrayList<>();
		String dsSql = "select bf.busin_id,bf.busin_code ,bf.busin_name ,p.id project_id,p.project_code,p.project_name "
				+ "from t_meta_business_field bf ,t_meta_project_ds ds ,t_meta_project p "
				+ "where bf.busin_id = p.busin_id  and ds.project_id = p.id and ds.ds_id = ?";
		String tabSql = "select table_id,table_code,db_type,ds_id from t_asset_meta_tab_collect where jdbc_url=? and db_code=?";
		String esSql = "select index_code as table_id,alias_name as table_code,'elasticsearch' as db_type,ds_id from t_asset_es_index_collect where ds_id = ?";
		List<Map<String, Object>> dsData;
		List<Map<String, Object>> tabData;
		for (Map<String, Object> map : struct) {
			dsData = gJdbc.queryForList(dsSql,map.get("ds_id"));
			if (String.valueOf(map.get("data_source_type")).equals("elasticsearch")) {
				tabData = gJdbc.queryForList(esSql,map.get("ds_id"));
			} else {
				tabData = gJdbc.queryForList(tabSql,map.get("jdbcUrl"),map.get("database"));
			}
			//清理结构关系
			gJdbc.update("delete from t_asset_meta_struct where ds_id =? ",map.get("ds_id"));
			
			for (Map<String, Object> ds : dsData) {
				String busin_id = String.valueOf(ds.get("busin_id"));
				String busin_code = String.valueOf(ds.get("busin_code"));
				String busin_name = String.valueOf(ds.get("busin_name"));
				String project_id = String.valueOf(ds.get("project_id"));
				String project_code = String.valueOf(ds.get("project_code"));
				String project_name = String.valueOf(ds.get("project_name"));
				Map<String, Object> structMap;
				for (Map<String, Object> tab : tabData) {
					structMap = new HashMap<String, Object>();
					structMap.put("busin_id", busin_id);
					structMap.put("busin_code", busin_code);
					structMap.put("busin_name", busin_name);
					structMap.put("project_id", project_id);
					structMap.put("project_code", project_code);
					structMap.put("project_name", project_name);
					structMap.put("table_id", tab.get("table_id"));
					structMap.put("table_code", tab.get("table_code"));
					structMap.put("db_type", tab.get("db_type"));
					structMap.put("ds_id", tab.get("ds_id"));
					structMap.put("db_code", map.get("database"));
					structData.add(structMap);
				}
			}
		};
		saveStructMeta(structData);
	}
	
	
	/**
	 * 合并hive，只合并表，单独加结构关系、日志
	 * @return 
	 */
	@Transactional
	private Map<String, Object> mergeHiveMeta(String projectCode) {
		Map<String, Object> logMap = new HashMap<>();
		try {
			
			if (null == projectCode || "".equals(projectCode)) {
				projectCode = "1=1";
			} else {
				projectCode = "t0.project_code ='" + projectCode +"'";
			}
			gJdbc.execute("delete from t_asset_meta_tab_collect where db_type='hive'");
			String sql = "select t0.object_type as tableType,'hive' as dbType,t0.project_code as dbCode,t0.cube_id as tableId,t0.cube_code as tableCode,t0.cube_name as tableName,t1.partitionColumn,ifnull(t1.rowNum,0) as rowNum,ifnull(t1.rowSize,0) as rowSize,t0.create_time as createTime,now() as lastSyncTime,t0.oper_user as createUser "
				+ "from t_meta_cubes t0 left join "
				+ "(select table_code,project_code,max(partition_column) as partitionColumn ,sum(num_rows) as rowNum,sum(hdfs_size) as rowSize "
				+ "from t_asset_sd_collect "
				+ "group by table_code,project_code) t1 on t0.project_code = t1.project_code and t0.cube_code = t1.table_code and t0.object_type in ('10', '11', '16', '17', '18','50','37','38','39','40','42','43') and " + projectCode;
 		
 			List<MetaTabBean> hiveList = gJdbc.query(sql,new BeanPropertyRowMapper<MetaTabBean>(MetaTabBean.class));
 			// 			hiveList.forEach(bean -> System.out.println(bean.getTableCode()));
			saveTabMeta(hiveList);
			
			//归属结构关系
			gJdbc.execute("delete from t_asset_meta_struct where db_type='hive'");
			sql = "select business_field_id as busin_id,business_field_name as busin_name,project_id,project_code,project_name,cube_id as table_id,cube_code as table_code,'hive' as db_type,project_code as db_code from t_meta_cubes t0 where " + projectCode;
			List<Map<String, Object>> structList = gJdbc.queryForList(sql);
			saveStructMeta(structList);
			
			logMap.put("db_type", "hive");
			logMap.put("table_num", hiveList.size());
			logMap.put("status", "0");
			return logMap;
		} catch (Exception e) {
			e.printStackTrace();
			logMap.put("db_type", "hive");
			logMap.put("err_log", e.getMessage());
			logMap.put("status", "1");
			return logMap;
		}
	}
	
    public static int FNVHash(String data) {
    	if (null == data) {
    		return 216613626;
    	}
        final int p = 1218;
        int hash = (int) 2166136261L;
        for (int i = 0; i < data.length(); i++)
               hash = (hash ^ data.charAt(i)) * p;
        hash += hash << 13;
        hash ^= hash >> 7;
        hash += hash << 3;
        hash ^= hash >> 17;
        hash += hash << 5;
        return hash;
 }
    
	 public static int SToA(String content){
		    int result = 0;
		    int max = content.length();
		    for (int i=0; i<max; i++){
		      char c = content.charAt(i);
		      int b = (int)c;
		      result = result + b;
		    }
		    return result;
		  }
	 
	 static long UUID(String str1,String str2,String str3) {
		 long id1 = Math.abs(FNVHash(str1))+SToA(str1);
		 long id2 = Math.abs(FNVHash(str2))+SToA(str2);
		 long id3 = Math.abs(FNVHash(str3))+SToA(str3);
		return id1+id2+id3;
	 }
	 
	 public List<Map<String, Object>> getTableInfo(Map<String, Object> params) {
		 String ds_id = String.valueOf(params.get("ds_id"));
		 String db_code = String.valueOf(params.get("db_code"));
		 String table_code = String.valueOf(params.get("table_code"));
		 String columns_code = String.valueOf(params.get("columns_code"));
		 String where = " and db_code='" + db_code + "' and table_code='" + table_code + "'";
		 if ("".equals(db_code) || "null".equals(db_code) || "".equals(table_code) || "null".equals(table_code)) {
			 where = " and 1=1";
		 }
		 if ("".equals(columns_code) || "null".equals(columns_code)) {
			 columns_code = " * ";
		 }
		 String sql = "select " + columns_code + " from t_asset_meta_tab_collect where ds_id=?" + where;
		 List<Map<String, Object>> tableData = gJdbc.queryForList(sql,ds_id);
		 return tableData;
	 }
	 
	 public List<Map<String, Object>> getColumnInfo(Map<String, Object> params) {
		 String ds_id = String.valueOf(params.get("ds_id"));
		 String db_code = String.valueOf(params.get("db_code"));
		 String table_code = String.valueOf(params.get("table_code"));
		 String columns_code = String.valueOf(params.get("columns_code"));
		 if ("".equals(columns_code) || "null".equals(columns_code)) {
			 columns_code = " c.* ";
		 } else {
			 String[] colArry = columns_code.trim().split(",");
			 columns_code = "";
			 for (int i=0; i<colArry.length; i++) {
				 columns_code = columns_code + ",c." + colArry[i];
			 }
			 columns_code = columns_code.substring(1);
		 }
		 String sql = "select " + columns_code + " from t_asset_meta_col_collect c,t_asset_meta_tab_collect t where c.table_id = t.table_id and t.ds_id=? and c.db_code=? and c.table_code=?";
		 List<Map<String, Object>> colData = gJdbc.queryForList(sql,ds_id,db_code,table_code);
		 return colData;
	 }
	 
	 public List<Map<String, Object>> getPartitionInfo(Map<String, Object> params) {
		 String ds_id = String.valueOf(params.get("ds_id"));
		 String db_code = String.valueOf(params.get("db_code"));
		 String table_code = String.valueOf(params.get("table_code"));
		 String columns_code = String.valueOf(params.get("columns_code"));
		 if ("".equals(columns_code) || "null".equals(columns_code)) {
			 columns_code = " p.* ";
		 } else {
			 String[] colArry = columns_code.trim().split(",");
			 columns_code = "";
			 for (int i=0; i<colArry.length; i++) {
				 columns_code = columns_code + ",p." + colArry[i];
			 }
			 columns_code = columns_code.substring(1);
		 }
		 String sql = "select " + columns_code + " from t_asset_meta_prt_collect p,t_asset_meta_tab_collect t where p.table_id = t.table_id and t.ds_id=? and p.db_code=? and p.table_code=?";
		 List<Map<String, Object>> colData = gJdbc.queryForList(sql,ds_id,db_code,table_code);
		 return colData;
	 } 
	
	public static void main(String[] args) throws SQLException, IOException {
		OkHttpClient client = new OkHttpClient();

		Request request = new Request.Builder()
//		  .url("http://172.30.132.191:29200/_cat/indices")
		  .url("http://172.30.132.167:19201/dp_dwd_access_detail_index/_search")
		  .get()
//		  .addHeader("Authorization", "Basic ZWxhc3RpYzpvNERVcnRxYXJXcVpnMjg5WEhZRw==")
//		  .addHeader("User-Agent", "PostmanRuntime/7.13.0")
		  .addHeader("Accept", "*/*")
		  .addHeader("Cache-Control", "no-cache")
//		  .addHeader("Postman-Token", "e6354adc-f43a-42da-97ec-b6da2533a11a,73ccb178-9e31-4652-8a1c-a3be5d076f66")
//		  .addHeader("Host", "172.30.132.191:29200")
//		  .addHeader("accept-encoding", "gzip, deflate")
		  .addHeader("Connection", "keep-alive")
		  .addHeader("cache-control", "no-cache")
		  .build();

		Response response = client.newCall(request).execute();
		System.out.println(response.body().string());
	}
}
